说明：¶
- 参照于http://cookdata.cn/note/view_static_note/84230091c4104946ab3d22dd675df58d/
- 前面部分爬取数据
- 后面部分可视化
In [ ]:
# 查看当前挂载的数据集目录, 该目录下的变更重启环境后会自动还原
# View dataset directory. This directory will be recovered automatically after resetting environment.
!ls /home/aistudio/data
In [ ]:
# 查看工作区文件, 该目录下的变更将会持久保存. 请及时清理不必要的文件, 避免加载过慢.
# View personal work directory. All changes under this directory will be kept even after reset. Please clean unnecessary files in time to speed up environment loading.
!ls /home/aistudio/work
In [2]:
# 如果需要进行持久化安装, 需要使用持久化路径, 如下方代码示例:
# If a persistence installation is required, you need to use the persistence path as the following:
!mkdir /home/aistudio/external-libraries
!pip install pyecharts -t /home/aistudio/external-libraries
In [3]:
# 同时添加如下代码, 这样每次环境(kernel)启动的时候只要运行下方代码即可:
# Also add the following code, so that every time the environment (kernel) starts, just run the following code:
import sys
sys.path.append('/home/aistudio/external-libraries')
In [1]:
#请求数据，导入使用的一些包
import requests
import pandas as pd
import time
import json
pd.set_option('max_rows', 500)
In [ ]:
#设置请求开头，伪装成浏览器
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
}
#发起请求，将找到的第一个数据源作为请求目标
url = 'https://c.m.163.com/ug/api/wuhan/app/data/list-total'
r = requests.get(url, headers=headers) # 使用requests发起请求
#根据网页内容，状态为200为成功
print(r.status_code)
print(type(r.text))
print(len(r.text))
#将数据转为json格式
data_json = json.loads(r.text)
data_json.keys()
#注意，从原网页上可以看出数据保存在data中
Out[ ]:
In [ ]:
#上面的keys中的data保存着需要的数据
data = data_json['data']
data.keys()
Out[ ]:
| keys | details |
|---|---|
| chinaTotal | 全国当日数据 |
| chinaDayList | 全国历史数据 |
| lastUpdateTime | 更新时间 |
| overseaLastUpdateTime | 海外更新时间 |
| areaTree | 世界各地实时数据 |
In [ ]:
#全国各省实时数据爬取
data_province = data['areaTree'][2]['children']
#数据为列表格式
data_province[0].keys()
Out[ ]:
| keys | details |
|---|---|
| today | 省当日数据 |
| total | 省当日累计数据 |
| extData | 无数据 |
| name | 省名 |
| id | 省编号 |
| lastUpdateTime | 更新时间 |
| children | 子数据 |
In [ ]:
#遍历查看各省名称、更新时间
for i in range(len(data_province)):
print(data_province[i]['name'], data_province[i]['lastUpdateTime'])
if i == 10:
break
不需要采集的数据：children、extData。 需要采集的数据：由于数据中today和total嵌套着字典，因此不能直接获取，对于id、lastUpdateTime、name、可以直接取出为一个数据，today为一个数据，total为一个数据，最后三个数据合并为一个数据。
In [ ]:
# 获取id、lastUpdateTime、name
info = pd.DataFrame(data_province)[['id','lastUpdateTime','name']]
info.head()
Out[ ]:
In [ ]:
# 将提取数据的方法封装为函数
def get_data(data,info_list):
info = pd.DataFrame(data)[info_list] # 主要信息
today_data = pd.DataFrame([i['today'] for i in data ]) # 生成today的数据
today_data.columns = ['today_'+i for i in today_data.columns] # 修改列名
total_data = pd.DataFrame([i['total'] for i in data ]) # 生成total的数据
total_data.columns = ['total_'+i for i in total_data.columns] # 修改列名
return pd.concat([info,total_data,today_data],axis=1) # info、today和total横向合并最终得到汇总的数据
today_province = get_data(data_province,['id','lastUpdateTime','name'])
today_province.head()
Out[ ]:
In [ ]:
def save_data(data,name): # 定义保存数据方法
file_name = name+'_'+time.strftime('%Y_%m_%d',time.localtime(time.time()))+'.csv'
data.to_csv(file_name,index=None,encoding='utf_8_sig')
print(file_name+' 保存成功！')
save_data(today_province,'today_province')
time.strftime('%Y_%m_%d',time.localtime(time.time()))
Out[ ]:
In [ ]:
#开始提取世界各国实时数据
areaTree = data['areaTree'] # 取出areaTree
#瞅瞅第一个国家
areaTree[0]
#和之前的差不多
Out[ ]:
In [ ]:
for i in range(len(areaTree)): # 查看各国家名称和更新时间
print(areaTree[i]['name'],areaTree[i]['lastUpdateTime'])
if i == 10:
break
In [ ]:
#从areaTree中提取每个国家的实时数据
today_world = get_data(areaTree,['id','lastUpdateTime','name'])
today_world.head()
Out[ ]:
In [ ]:
#保存世界各国的
save_data(today_world,'today_world')
历史数据的爬取¶
In [ ]:
chinaDayList = data['chinaDayList']
chinaDayList[0]
Out[ ]:
In [ ]:
alltime_China = get_data(chinaDayList,['date','lastUpdateTime'])
alltime_China.head()
Out[ ]:
In [ ]:
#全国历史数据
save_data(alltime_China,'alltime_China')
In [ ]:
#省历史数据
province_dict = {num:name for num,name in zip(today_province['id'],today_province['name'])}
count = 0
for i in province_dict:
print(i,province_dict[i])
count += 1
if count == 10:
break
In [ ]:
#得到每个省的数据并合并
start = time.time()
for province_id in province_dict: # 遍历各省编号
try:
# 按照省编号访问每个省的数据地址，并获取json数据
url = 'https://c.m.163.com/ug/api/wuhan/app/data/list-by-area-code?areaCode='+province_id
r = requests.get(url, headers=headers)
data_json = json.loads(r.text)
# 提取各省数据，然后写入各省名称
province_data = get_data(data_json['data']['list'],['date'])
province_data['name'] = province_dict[province_id]
# 合并数据
if province_id == '420000':
alltime_province = province_data
else:
alltime_province = pd.concat([alltime_province,province_data])
print('-'*20,province_dict[province_id],'成功',
province_data.shape,alltime_province.shape,
',累计耗时:',round(time.time()-start),'-'*20)
# 设置延迟等待
time.sleep(20)
except:
print('-'*20,province_dict[province_id],'wrong','-'*20)
In [ ]:
alltime_province.info()
In [ ]:
save_data(alltime_province,'alltime_province')
世界各国历史数据爬取¶
In [ ]:
country_dict = {key:value for key,value in zip(today_world['id'], today_world['name'])}
# 查看前十个内容
count = 0
for i in country_dict:
print(i,country_dict[i])
count += 1
if count == 10:
break
In [ ]:
start = time.time()
for country_id in country_dict: # 遍历每个国家的编号
try:
# 按照编号访问每个国家的数据地址，并获取json数据
url = 'https://c.m.163.com/ug/api/wuhan/app/data/list-by-area-code?areaCode='+country_id
r = requests.get(url, headers=headers)
json_data = json.loads(r.text)
# 生成每个国家的数据
country_data = get_data(json_data['data']['list'],['date'])
country_data['name'] = country_dict[country_id]
# 数据叠加
if country_id == '9577772':
alltime_world = country_data
else:
alltime_world = pd.concat([alltime_world,country_data])
print('-'*20,country_dict[country_id],'成功',country_data.shape,alltime_world.shape,
',累计耗时:',round(time.time()-start),'-'*20)
time.sleep(25)
except:
print('-'*20,country_dict[country_id],'wrong','-'*20)
print('end')
In [ ]:
print(alltime_world.shape)
save_data(alltime_world,'alltime_world')
到这里为止，数据爬取已经完成了，接下来是数据可视化¶
In [4]:
import pandas as pd
import pyecharts
import pyecharts.options as opts
from pyecharts.charts import Map
In [5]:
#读取数据
world_data = pd.read_csv('today_world_2020_05_25.csv')
world_data.keys()
Out[5]:
In [6]:
#现存确诊人数，并添加到数据中
world_data['today_storeConfirm'] = world_data['total_confirm'] - world_data['total_heal'] - world_data['total_dead']
In [7]:
world_data.head()
Out[7]:
In [8]:
#中英文对照
contry_name = pd.read_csv('name.csv')
contry_name.head()
Out[8]:
In [9]:
world_data['eg_name'] = world_data['name'].replace(contry_name['中文'].values ,contry_name['英文'].values)
world_data['eg_name'].head()
Out[9]:
In [10]:
#提取出我们需要的数据，保存成一个嵌套列表的形式：
heatmap_data = world_data[['eg_name','today_storeConfirm']].values.tolist()
heatmap_data[:10]
Out[10]:
In [11]:
#接下来我们开始绘图，首先初始化类对象Map，并调用add方法添加绘图基本配置：
map_ = Map().add(series_name = "今日现存确诊人数", # 设置提示框标签
data_pair = heatmap_data, # 输入数据
maptype = "world", # 设置地图类型为世界地图
is_map_symbol_show = False # 不显示标记点
)
# 设置系列配置项
map_.set_series_opts(label_opts=opts.LabelOpts(is_show=False)) # 不显示国家（标签）名称
# 设置全局配置项
map_.set_global_opts(title_opts = opts.TitleOpts(title="世界各国今日现存确诊人数地图"), # 设置图标题
# 设置视觉映射配置项
visualmap_opts = opts.VisualMapOpts(pieces=[ # 自定义分组的分点和颜色
{"min": 10000,"color":"#800000"}, # 栗色
{"min": 5000, "max": 9999, "color":"#B22222"}, # 耐火砖
{"min": 999, "max": 4999,"color":"#CD5C5C"}, # 印度红
{"min": 100, "max": 999, "color":"#BC8F8F"}, # 玫瑰棕色
{"max": 99, "color":"#FFE4E1"}, # 薄雾玫瑰
],
is_piecewise = True)) # 显示分段式图例
# 在notebook中进行渲染
map_.render_notebook()
Out[11]:
世界国家累计死亡人数玫瑰图¶
In [12]:
#先筛选死亡大于1000的国家，再降序排序
need_data = world_data[['name','total_dead']][world_data['total_dead'] >1000]
rank = need_data[['name','total_dead']].sort_values(by='total_dead',ascending=False).values
In [13]:
#导入Pie
from pyecharts.charts import Pie
pie = Pie().add("累计死亡人数", # 添加提示框标签
rank, # 输入数据
radius = ["20%", "80%"], # 设置内半径和外半径
center = ["60%", "60%"], # 设置圆心位置
rosetype = "radius") # 玫瑰图模式，通过半径区分数值大小，角度大小表示占比
pie.set_global_opts(title_opts = opts.TitleOpts(title="世界国家累计死亡人数玫瑰图", # 设置图标题
pos_right = '40%'), # 图标题的位置
legend_opts = opts.LegendOpts( # 设置图例
orient='vertical', # 垂直放置图例
pos_right="85%", # 设置图例位置
pos_top="15%"))
pie.set_series_opts(label_opts = opts.LabelOpts(formatter="{b} : {d}%")) # 设置标签文字形式为（国家：占比（%））
# 在notebook中进行渲染
pie.render_notebook()
Out[13]:
3月以来世界国家累计确诊人数动态条形图¶
In [14]:
alltime_data = pd.read_csv('alltime_world_2020_05_25.csv')
#首先挑选出疫情最为严重的一些国家，并筛选出这些国家的历史疫情数据：
country_list = ['美国', '英国', '意大利', '西班牙', '法国', '巴西', '比利时', '德国', '伊朗','墨西哥', '加拿大', '荷兰', '中国']
need_data = alltime_data[alltime_data['name'].isin(country_list)]
In [15]:
#使用datetime模块生成时间数据，构造时间列表：
from datetime import datetime,timedelta
time_list1 = [(datetime(2020, 3, 1) + timedelta(i)).strftime('%Y-%m-%d') for i in range(31)]
time_list2 = [(datetime(2020, 4, 1) + timedelta(i)).strftime('%Y-%m-%d') for i in range(30)]
time_list3 = [(datetime(2020, 5, 1) + timedelta(i)).strftime('%Y-%m-%d') for i in range(25)]
time_list = time_list1 + time_list2 + time_list3
print(time_list)
In [16]:
day = time_list[0]
draw_data = need_data[need_data['date']==day][['name','total_confirm']].sort_values(by='total_confirm',ascending=True)
print(draw_data)
print(draw_data['name'].replace(contry_name['中文'].values ,contry_name['英文'].values))
In [17]:
import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['figure.dpi'] = 100
In [18]:
#设置颜色
color_list = ['brown','peru','orange','blue','green',
'red','yellow','teal','pink','orchid', 'cornflowerblue', 'darkslategray', 'lightcyan']
country_color = pd.DataFrame()
country_color['country'] = country_list
country_color['color'] = color_list
In [19]:
#定义绘图函数
import matplotlib.ticker as ticker
def barh_draw(day):
# 提取每一天的数据
draw_data = need_data[need_data['date']==day][['name','total_confirm']].sort_values(by='total_confirm',ascending=True)
# 清空当前的绘图
ax.clear()
# 绘制条形图
ax.barh(draw_data['name'].replace(contry_name['中文'].values ,contry_name['英文'].values),draw_data['total_confirm'], color=[country_color[country_color['country']==i]['color'].values[0] for i in draw_data['name']])
# 数值标签的间距
dx = draw_data['total_confirm'].max()/200
# 添加数值标签
for j, (name, value) in enumerate(zip(draw_data['name'], draw_data['total_confirm'])):
ax.text(value+dx, j, f'{value:,.0f}', size=10, ha='left', va='center')
# 添加日期标签
ax.text(draw_data['total_confirm'].max()*0.75, 0.4, day, color='#777777',size=40, ha='left')
# 设置刻度标签的格式
ax.xaxis.set_major_formatter(ticker.StrMethodFormatter('{x:,.0f}'))
# 设置刻度的位置
ax.xaxis.set_ticks_position('top')
# 设置刻度标签的颜色和大小
ax.tick_params(axis='x',colors='#777777', labelsize=15)
# 添加网格线
ax.grid(which='major',axis='x',linestyle='-')
# 添加图标题
ax.text(0, 13, 'The cumulative number of confirmed cases in countries around the world',size=20, ha='left')
# 去除图边框
plt.box(False)
# 关闭绘图框
plt.close()
In [20]:
fig, ax = plt.subplots(figsize=(14, 8))
import matplotlib.animation as animation
from IPython.display import HTML
animator = animation.FuncAnimation(fig, barh_draw, frames=time_list, interval=200)
HTML(animator.to_jshtml())
Out[20]: